{
  "cells": [
    {
      "cell_type": "code",
      "execution_count": 46,
      "metadata": {
        "collapsed": true,
        "pycharm": {
          "is_executing": false
        }
      },
      "outputs": [],
      "source": "import numpy as np\nW \u003d np.random.random((8,1))\nalpha \u003d 0.01\ngamma \u003d 0.99\n\ndef S(i):\n    return W[i]*2+W[7]\n\ndef greedy_action():  \n    return np.argmax(W)"
    },
    {
      "cell_type": "code",
      "execution_count": 47,
      "outputs": [],
      "source": "def dashed():\n    dice \u003d np.random.randint(low\u003d0, high\u003d6)\n    return dice\n\ndef solid():\n    return 7\n\ndef b():\n    dice \u003d np.random.random()\n    if dice \u003e 1/7:\n        return dashed()\n    else:\n        return solid()\n    \ndef rho(dest):\n    if dest \u003d\u003d 7:\n        return 7\n    else:\n        return 0",
      "metadata": {
        "pycharm": {
          "metadata": false,
          "name": "#%%\n",
          "is_executing": false
        }
      }
    },
    {
      "cell_type": "code",
      "execution_count": 48,
      "outputs": [],
      "source": "def update(action):\n    global W\n    # delta \u003d reward(\u003d0) + gamma * greedyQ(S+t+1, greedy action. w) -  Q(S,a,w)\n    # Because Baird\u0027s special setting, we only need to care the destination of action\n    # to determine the Q, which should be equal to the next state value(one step)\n    delta \u003d gamma*S(greedy_action()) - gamma*S(action)\n    W \u003d W + alpha*delta*rho(action)*W\n    \ndef game():\n    global W\n    for episode in range(1,31):\n        W \u003d np.random.random((8,1))\n        for t in range(1,1000):\n            action \u003d b()\n            update(action)\n            if max(W) \u003d\u003d np.inf:\n                print(\u0027Episode\u0027,episode,\u0027diverged at step\u0027, t)\n                break",
      "metadata": {
        "pycharm": {
          "metadata": false,
          "name": "#%%\n",
          "is_executing": false
        }
      }
    },
    {
      "cell_type": "code",
      "execution_count": 49,
      "outputs": [
        {
          "name": "stdout",
          "text": [
            "Episode 1 diverged at step 133\nEpisode 2 diverged at step 302\nEpisode",
            " 3 diverged at step 238\nEpisode 5 diverged at step 193\nEpisode 7 diverged at step 229\nEpisode 9 diverged at step 235\nEpisode 10 diverged at step 335\nEpisode 11 diverged at step 257\nEpisode 12 diverged at step 284\nEpisode 13 diverged at step 155\nEpisode 14 diverged at step 134\nEpisode 15 diverged at step 114\nEpisode 16 diverged at step 523\nEpisode 17 diverged at step 204\nEpisode 18 diverged at step 218\nEpisode 19 diverged at step 185\nEpisode",
            " 20 diverged at step 693\nEpisode 21 diverged at step 182\nEpisode 22 diverged at step 633\nEpisode 23 diverged at step 130\nEpisode 25 diverged at step 187\nEpisode 26 diverged at step 153\nEpisode 28 diverged at step 361\nEpisode 29 diverged at step 178\nEpisode 30 diverged at step 392\n"
          ],
          "output_type": "stream"
        },
        {
          "name": "stderr",
          "text": [
            "/Users/yifanwang/anaconda3/envs/GYM_Practice/lib/python3.7/site-packages/ipykernel_launcher.py:7: RuntimeWarning: overflow encountered in multiply\n  import sys\n/Users/yifanwang/anaconda3/envs/GYM_Practice/lib/python3.7/site-packages/ipykernel_launcher.py:7: RuntimeWarning: overflow encountered in add\n  import sys\n/Users/yifanwang/anaconda3/envs/GYM_Practice/lib/python3.7/site-packages/ipykernel_launcher.py:6: RuntimeWarning: invalid value encountered in subtract\n  \n"
          ],
          "output_type": "stream"
        }
      ],
      "source": "game()\n",
      "metadata": {
        "pycharm": {
          "metadata": false,
          "name": "#%%\n",
          "is_executing": false
        }
      }
    }
  ],
  "metadata": {
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 2
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython2",
      "version": "2.7.6"
    },
    "kernelspec": {
      "name": "python3",
      "language": "python",
      "display_name": "Python 3"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}
